package fetcher

import (
	"bufio"
	"fmt"
	"golang.org/x/net/html/charset"
	"golang.org/x/text/encoding"
	"golang.org/x/text/encoding/unicode"
	"golang.org/x/text/transform"
	"io/ioutil"
	"net/http"
)

// 爬去网页内容
func Fetch(url string) ([]byte, error) {

	resp, err := getRequest(url)
	if err != nil {
		return nil, err
	}
	defer resp.Body.Close()

	if resp.StatusCode != http.StatusOK {
		return nil, fmt.Errorf("抓取出错了, 返回码[%d]\n", resp.StatusCode)
	}

	// 将流转换为utf8编码
	r := bufio.NewReader(resp.Body)
	e := determineEncoding(r)
	utf8Reader := transform.NewReader(r, e.NewDecoder())
	body, err := ioutil.ReadAll(utf8Reader)

	return body, err
}

func getRequest(url string) (*http.Response, error) {
	client := &http.Client{}
	req, _ := http.NewRequest("get", url, nil)

	req.Header.Set("User-Agent", "Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/62.0.3202.62 Safari/537.36")

	return client.Do(req)
}

// 从给定的输入流获取其编码格式
func determineEncoding(b *bufio.Reader) encoding.Encoding {
	data, err := b.Peek(1024)
	if err != nil {
		return unicode.UTF8
	}

	e, _, _ := charset.DetermineEncoding(data, "")

	return e
}
